{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chatbot Tutorial\n",
    "\n",
    "## 1. Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "from __future__ import unicode_literals\n",
    "\n",
    "import torch\n",
    "from torch.jit import script, trace\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import csv\n",
    "import random\n",
    "import re\n",
    "import os\n",
    "import unicodedata\n",
    "import codecs\n",
    "from io import open\n",
    "import itertools\n",
    "import math\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if USE_CUDA else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load & Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_name = 'cornell_movie_dialogs_corpus'\n",
    "corpus = os.path.join('data', corpus_name)\n",
    "\n",
    "def printLines(file, n=10):\n",
    "    with open(file, 'rb') as fr:\n",
    "        lines = fr.readlines()\n",
    "    for line in lines[:n]:\n",
    "        print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'L1045 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ They do not!\\n'\n",
      "b'L1044 +++$+++ u2 +++$+++ m0 +++$+++ CAMERON +++$+++ They do to!\\n'\n",
      "b'L985 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ I hope so.\\n'\n",
      "b'L984 +++$+++ u2 +++$+++ m0 +++$+++ CAMERON +++$+++ She okay?\\n'\n",
      "b\"L925 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ Let's go.\\n\"\n",
      "b'L924 +++$+++ u2 +++$+++ m0 +++$+++ CAMERON +++$+++ Wow\\n'\n",
      "b\"L872 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ Okay -- you're gonna need to learn how to lie.\\n\"\n",
      "b'L871 +++$+++ u2 +++$+++ m0 +++$+++ CAMERON +++$+++ No\\n'\n",
      "b'L870 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ I\\'m kidding.  You know how sometimes you just become this \"persona\"?  And you don\\'t know how to quit?\\n'\n",
      "b'L869 +++$+++ u0 +++$+++ m0 +++$+++ BIANCA +++$+++ Like my fear of wearing pastels?\\n'\n"
     ]
    }
   ],
   "source": [
    "printLines(os.path.join(corpus, 'movie_lines.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create formatted data file\n",
    "\n",
    "Formatted data file: each line contains ***a tab-separated query sentence*** and ***a response sentence*** pair."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splits each line into a dictionary: \n",
    "# lines是字典，每个元素line：key=lineID, value=lineObj={lineID:xxx, characterID:xxx, movieID:xxx, character:xxx, text:xxx}\n",
    "def loadLines(file, cols):\n",
    "    lines = {}\n",
    "    with open(file, 'r', encoding='iso-8859-1') as fr:\n",
    "        for line in fr:\n",
    "            values = line.split(' +++$+++ ')\n",
    "            lineObj = {col: values[i] for i, col in enumerate(cols)}\n",
    "            lines[lineObj['lineID']] = lineObj\n",
    "    return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Groups cols of lines from `loadLines` into conversations based on movie_conversations.txt\n",
    "# conversations是列表，每个元素convObj: {col1:xxx, col2:xxx, ..., lines: [lineObj1, lineObj2, ..., lineObjm]}\n",
    "def loadConversations(file, lines, cols):\n",
    "    conversations = []\n",
    "    with open(file, 'r', encoding='iso-8859-1') as fr:\n",
    "        for line in fr:\n",
    "            values = line.split(' +++$+++ ')\n",
    "            convObj = {col: values[i] for i, col in enumerate(cols)}\n",
    "            convObj['lines'] = [lines[lineId] for lineId in eval(convObj['utteranceIDs'])]\n",
    "            conversations.append(convObj)\n",
    "    return conversations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> 刘尧：训练数据是每个conversation中所有sentence生成的sentece对: <前一句话, 后一句话\\>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracts pairs of sentences from conversations\n",
    "# qa_pair是列表，每个元素是sentence pair: [conv1_text1,conv1_text2], [1_2,1_3], [1_3,1_4], [1_4,1_5], ..., [2_1,2_2], [2_2,2_3], ...\n",
    "def extractSentencePairs(conversations):\n",
    "    qa_pair = []\n",
    "    for conv in conversations:\n",
    "        for i in range(len(conv['lines']) - 1):  # Ignore the last line (no answer for it)\n",
    "            inputLine = conv['lines'][i]['text'].strip()\n",
    "            targetLine = conv['lines'][i + 1]['text'].strip()\n",
    "            if inputLine and targetLine:\n",
    "                qa_pair.append([inputLine, targetLine])\n",
    "    return qa_pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "datafile = os.path.join(corpus, 'formatted_movie_lines.txt')\n",
    "delimiter = '\\t'\n",
    "delimiter = str(codecs.decode(delimiter, 'unicode_escape'))  # 有时间好好研究这句话！！！\n",
    "\n",
    "lines = {}\n",
    "conversations = []\n",
    "MOVIE_LINES_COLS = ['lineID', 'characterID', 'movieID', 'character', 'text']\n",
    "MOVIE_CONVERSATIONS_COLS = ['character1ID', 'character2ID', 'movieID', 'utteranceIDs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing corpus and loading conversations...\n"
     ]
    }
   ],
   "source": [
    "print('\\nProcessing corpus and loading conversations...')\n",
    "lines = loadLines(os.path.join(corpus, 'movie_lines.txt'), MOVIE_LINES_COLS)\n",
    "conversations = loadConversations(os.path.join(corpus, 'movie_conversations.txt'), lines, MOVIE_CONVERSATIONS_COLS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Writing newly formatted file ...\n",
      "\n",
      "Sample lines from file: \n",
      "b\"Can we make this quick?  Roxanne Korrine and Andrew Barrett are having an incredibly horrendous public break- up on the quad.  Again.\\tWell, I thought we'd start with pronunciation, if that's okay with you.\\n\"\n",
      "b\"Well, I thought we'd start with pronunciation, if that's okay with you.\\tNot the hacking and gagging and spitting part.  Please.\\n\"\n",
      "b\"Not the hacking and gagging and spitting part.  Please.\\tOkay... then how 'bout we try out some French cuisine.  Saturday?  Night?\\n\"\n",
      "b\"You're asking me out.  That's so cute. What's your name again?\\tForget it.\\n\"\n",
      "b\"No, no, it's my fault -- we didn't have a proper introduction ---\\tCameron.\\n\"\n",
      "b\"Cameron.\\tThe thing is, Cameron -- I'm at the mercy of a particularly hideous breed of loser.  My sister.  I can't date until she does.\\n\"\n",
      "b\"The thing is, Cameron -- I'm at the mercy of a particularly hideous breed of loser.  My sister.  I can't date until she does.\\tSeems like she could get a date easy enough...\\n\"\n",
      "b'Why?\\tUnsolved mystery.  She used to be really popular when she started high school, then it was just like she got sick of it or something.\\n'\n",
      "b\"Unsolved mystery.  She used to be really popular when she started high school, then it was just like she got sick of it or something.\\tThat's a shame.\\n\"\n",
      "b'Gosh, if only we could find Kat a boyfriend...\\tLet me see what I can do.\\n'\n"
     ]
    }
   ],
   "source": [
    "print('\\nWriting newly formatted file ...')\n",
    "with open(datafile, 'w', encoding='utf8') as fw:\n",
    "    writer = csv.writer(fw, delimiter=delimiter, lineterminator='\\n')\n",
    "    for pair in extractSentencePairs(conversations):\n",
    "        writer.writerow(pair)\n",
    "print('\\nSample lines from file: ')\n",
    "printLines(datafile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and trim data\n",
    "\n",
    "> 刘尧：把vocabulary及其附属或衍生变量以及相关method封装抽象成Class！这样既保护数据又方便使用！好好好！！！\n",
    "\n",
    "Note that we are dealing with sequences of **words**, we should create a **vocabulary**: mapping each unique word that we encounter in our dataset to an index value.\n",
    "\n",
    "For this we define a ***Vocabulary*** class, which has 5 attributes and 3 methods:\n",
    "\n",
    "- 5 attributes\n",
    "\n",
    "    - **word2index**: A mapping from each word to index\n",
    "\n",
    "    - **index2word**: A reverse mapping from index to each word\n",
    "\n",
    "    - **word2count**: A mapping from each word to its count\n",
    "\n",
    "    - num_words: A total word count\n",
    "    \n",
    "    - trimmed: If infrequently seen words are trimmed\n",
    "\n",
    "- 3 methods\n",
    "\n",
    "    - **addWord**: Adding a word to the vacabulary\n",
    "\n",
    "    - addSentence: Adding all words in a sentence\n",
    "\n",
    "    - trim: Trimming infrequently seen words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Default word tokens\n",
    "PAD_TOKEN = 0\n",
    "SOS_TOKEN = 1  # Start of sentence\n",
    "EOS_TOKEN = 2  # End of sentence \n",
    "\n",
    "MAX_LENGTH = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.trimmed = False\n",
    "        self.word2index = {}  # 不默认包含PAD,SOS,EOS这仨\n",
    "        self.word2count = {}\n",
    "        self.index2word = {PAD_TOKEN: 'PAD', SOS_TOKEN: 'SOS', EOS_TOKEN: 'EOS'}\n",
    "        self.num_words = 3  # SOS, EOS, PAD\n",
    "        \n",
    "    def addWord(self, word):\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.num_words    # 添加的word，其index依次往后排\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.num_words] = word\n",
    "            self.num_words += 1\n",
    "        else:\n",
    "            self.word2count[word] += 1\n",
    "    \n",
    "    def addSentence(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            self.addWord(word)\n",
    "            \n",
    "    # Remove words below a certain count threshold\n",
    "    def trim(self, min_count):\n",
    "        if self.trimmed:\n",
    "            return\n",
    "        \n",
    "        keep_words = [k for k, v in self.word2count.items() if v >= min_count]\n",
    "        print(f'keep_words {len(keep_words)} / {len(self.word2index)} = {len(keep_words) / len(self.word2index): .4f}')\n",
    "        \n",
    "        # Reinitializa dictionaries\n",
    "        self.word2index = {}\n",
    "        self.word2count = {}\n",
    "        self.index2word = {PAD_TOKEN: 'PAD', SOS_TOKEN: 'SOS', EOS_TOKEN: 'EOS'}\n",
    "        self.num_words = 3\n",
    "        \n",
    "        for word in keep_words:\n",
    "            self.addWord(word)\n",
    "            \n",
    "        self.trimmed = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some data preprocessing:\n",
    "\n",
    "- **unicodeToAscii**: Convert the Unicode strings to ASCII\n",
    "\n",
    "- **normalizeString**: Convert all letters to lowercase and trim all non-letter characters except for basic punctuation\n",
    "\n",
    "- **filterPairs**: Filter sentences with length greater than the *MAX_LENGTH* threshold\n",
    "\n",
    "> 刘尧：这些常规的预处理，最好封装成一个个function，以方便使用！可以放在**Coding通用工具脚本**里！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unicodeToAscii(s):\n",
    "    \"\"\"Turn a Unicode string to plain ASCII, thanks to https://stackoverflow.com/a/518232/2809427\"\"\"\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalizeString(s):\n",
    "    \"\"\"Lowercase, trim, and remove non-letter characters\"\"\"\n",
    "    s = unicodeToAscii(s.lower().strip())\n",
    "    s = re.sub(r'([.!?])', r' \\1', s)      # 把.!?三个标点符号替换为？\n",
    "    s = re.sub(r'[^a-zA-Z.!?]+', r' ', s)  # 把字母和.!?之外的character替换为空格\n",
    "    s = re.sub(r'\\s+', r' ', s).strip()    # 把替换为空格\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readVocs(datafile, corpus_name):\n",
    "    \"\"\"Read <query, response> pairs and return a Vocabulary object\"\"\"\n",
    "    lines = open(datafile, encoding='utf8').read().strip().split('\\n')\n",
    "    pairs = [[normalizeString(s) for s in line.split('\\t')] for line in lines]\n",
    "    voc = Vocabulary(corpus_name)\n",
    "    return voc, pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterPair(pair):\n",
    "    \"\"\"Return True iff both sentences in pair are under the MAX_LENGTH threshold\"\"\"\n",
    "    return len(pair[0].split(' ')) < MAX_LENGTH and len(pair[1].split(' ')) < MAX_LENGTH\n",
    "\n",
    "def filterPairs(pairs):\n",
    "    \"\"\"Filter pairs using filterPair function\"\"\"\n",
    "    return [pair for pair in pairs if filterPair(pair)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadPrepareData(corpus, corpus_name, datafile, save_dir):  # corpus, save_dir 在哪里使用的！？！\n",
    "    \"\"\"Using the functions above, return a populated Vocabulary object and pairs list\"\"\"\n",
    "    print('Start preparing training data ...')\n",
    "    vocabulary, pairs = readVocs(datafile, corpus_name)\n",
    "    print('Read {!s} sentence pairs'.format(len(pairs)))\n",
    "    pairs = filterPairs(pairs)\n",
    "    print('Trimmed to {!s} sentence pairs'.format(len(pairs)))\n",
    "    print('Counting words ...')\n",
    "    for pair in pairs:\n",
    "        vocabulary.addSentence(pair[0])\n",
    "        vocabulary.addSentence(pair[1])\n",
    "    print('Counted words: ', vocabulary.num_words)\n",
    "    return vocabulary, pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start preparing training data ...\n",
      "Read 221282 sentence pairs\n",
      "Trimmed to 64271 sentence pairs\n",
      "Counting words ...\n",
      "Counted words:  18008\n"
     ]
    }
   ],
   "source": [
    "# Load/Assemble vocabulary and pairs\n",
    "save_dir = os.path.join('data', 'save')\n",
    "vocabulary, pairs = loadPrepareData(corpus, corpus_name, datafile, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['there .', 'where ?']\n",
      "['you have my word . as a gentleman', 'you re sweet .']\n",
      "['hi .', 'looks like things worked out tonight huh ?']\n",
      "['you know chastity ?', 'i believe we share an art instructor']\n",
      "['have fun tonight ?', 'tons']\n",
      "['well no . . .', 'then that s all you had to say .']\n",
      "['then that s all you had to say .', 'but']\n",
      "['but', 'you always been this selfish ?']\n",
      "['do you listen to this crap ?', 'what crap ?']\n",
      "['what good stuff ?', 'the real you .']\n"
     ]
    }
   ],
   "source": [
    "for pair in pairs[:10]:\n",
    "    print(pair)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another tactic that is beneficial to achieving faster convergence during training is **trimming rarely used words out of our vocabulary**. \n",
    "\n",
    "Decreasing the feature space will also soften the difficulty of the function that the model must learn to approximate.\n",
    "\n",
    "We will do this as a two-stage process:\n",
    "    \n",
    "- Trim words used under *MIN_COUNT* threshold using the *Vocabulary.trim* function\n",
    "\n",
    "- Filter out pairs with trimmed words\n",
    "\n",
    "> 刘尧：事先从Vocabulary中定义并删除不常见的word，即**OOV的word**，随后从训练数据中删除这些OOV的word！ \n",
    "\n",
    "> 刘尧：疑问：模型应用时遇到OOV的word咋办？？？跟训练一样，应用前也先使用trimRareWords来处理一下！？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIN_COUNT = 3\n",
    "def trimRareWords(vocabulary, pairs, MIN_COUNT):\n",
    "    \"\"\"基于MIN_COUNT，删除vocabulary中不常见的word，并从训练/应用数据中删除带有不常见word的pairs\"\"\"\n",
    "    vocabulary.trim(MIN_COUNT)\n",
    "    \n",
    "    keep_pairs = []\n",
    "    for pair in pairs:\n",
    "        input_sentence = pair[0]\n",
    "        output_sentence = pair[1]\n",
    "        keep_input, keep_output = True, True\n",
    "        \n",
    "        # 判断pairs中2个句子中是否存在OOV的word，一旦存在，则删除当前pairs\n",
    "        for word in input_sentence.split(' '):\n",
    "            if word not in vocabulary.word2index:\n",
    "                keep_input = False\n",
    "                break\n",
    "        for word in output_sentence.split(' '):\n",
    "            if word not in vocabulary.word2index:\n",
    "                keep_output = False\n",
    "                break\n",
    "        if keep_input and keep_output:\n",
    "            keep_pairs.append(pair)\n",
    "            \n",
    "    print(f'Trimmed from {len(pairs)} pairs to {len(keep_pairs)}, {len(keep_pairs) / len(pairs): .4f} of total')\n",
    "    return keep_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keep_words 7823 / 18005 =  0.4345\n",
      "Trimmed from 64271 pairs to 53165,  0.8272 of total\n"
     ]
    }
   ],
   "source": [
    "pairs = trimRareWords(vocabulary, pairs, MIN_COUNT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Prepare Data for Models\n",
    "\n",
    "Already done: preparing and massaging data into a nice vocabulary object and list of sentence pairs.\n",
    "\n",
    "Not done yet: preparing numerical torch tensors as model inputs.\n",
    "\n",
    "When batch_size=1, all we have to do is convert words in sentence pairs to their corresponding indexes from vocabulary and feed this to the models\n",
    "\n",
    "When batch_size>1, which can **speed up training and/or leverage GPU parallelization capabilities**, we must be mindful of the variation of sentence length in our batches. \n",
    "\n",
    "To accommodate sentences of different sizes in the same batch, we will make our batched input tensor of shape *(max_length, batch_size)*.\n",
    "\n",
    "If we simply convert sentences to tensors by converting words to their indexes (*indexesFromSentence*) and zero-pad, our tensor would have shape (*batch_size, max_length*). However, we need to be able to **index our batch along time(为啥子？)**, and **across all sequences in the batch**. \n",
    "\n",
    "Therefore, we transpose our input batch shape to (*max_length, batch_size*), so that indexing across the 1st dimension returns **a time step across all sentences** in the batch (*zeroPadding*).\n",
    "\n",
    "> Shape Transpose: (*batch_size, max_length*) --> (*max_length, batch_size*)\n",
    "\n",
    "![001](./image/seq2seq_batches.png)\n",
    "\n",
    "> 刘尧：这么一大段到底要讲个啥子玩意儿？！为什么一定要Transpose一下？！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentenceToIndexes(vocabulary, sentence):\n",
    "    \"\"\"sentence向量化\"\"\"\n",
    "    return [vocabulary.word2index[word] for word in sentence.split(' ')] + [EOS_TOKEN]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> 刘尧：疑惑：zeroPadding函数中，取seq_batch中的max_length_batch进行padding，那么不同batch的max_length_batch应该无法保证一样吧？\n",
    "\n",
    "> 刘尧：疑惑：或者本来就不必一样了，因为经Transpose之后，max_length_batch为第1维，**batch_size变成第2维，所有batch的batch_size必然一样**，所以可直接输入到network中！？\n",
    "\n",
    "> 刘尧：疑惑：以上是不是就是进行Transpose的重要原因！？！貌似与Keras或其他案例中不一样哎？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def zeroPadding(seq_batch, fillvalue=PAD_TOKEN):\n",
    "    \"\"\"按seq_batch中最长的seq的长度进行zero-padding，随后进行Transpose\"\"\"\n",
    "    return list(itertools.zip_longest(*seq_batch, fillvalue=fillvalue))  # 内部隐含了Transpose操作！！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seqsToMask2D(seqs, value=PAD_TOKEN):\n",
    "    \"\"\"判断seqs中每个seq的每个token是否取值为value(是=0 否=1)，把seqs转化为0和1的二维mask\"\"\"\n",
    "    return [[int(token != value) for token in seq] for seq in seqs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以下三个函数 *tempVar, inputVar, outputVar* 用于转化 sentence 数据：最普通的 sentences (list of sentence) --> Tensor of shape (batch_size, max_length_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentencesToTensor(sentences, vocabulary):\n",
    "    \"\"\"对序列中每个sentence进行向量化和0-padding操作，随后整个序列转化为Tensor\"\"\"\n",
    "    indexes_batch = [sentenceToIndexes(vocabulary, sentence) for sentence in sentences]  # 同一batch中的sentence向量化\n",
    "    padList = zeroPadding(indexes_batch)    # zero-pad同一batch中的向量\n",
    "    padVar = torch.LongTensor(padList)      # torch.LongTensor: dtype=torch.int64/torch.long所对应的CPU Tensor\n",
    "    return indexes_batch, padList, padVar\n",
    "\n",
    "def inputToTensor(sentences, vocabulary):\n",
    "    \"\"\"Returns padded input sequence tensor and lengths\"\"\"\n",
    "    indexes_batch, padList, padVar = sentencesToTensor(sentences, vocabulary)\n",
    "    lengths = torch.tensor([len(indexes) for indexes in indexes_batch])\n",
    "    return padVar, lengths\n",
    "\n",
    "def targetToTensor(sentences, vocabulary):\n",
    "    \"\"\"Returns padded target sequence tensor, padding mask, and max target length\"\"\"\n",
    "    indexes_batch, padList, padVar = sentencesToTensor(sentences, vocabulary)\n",
    "    max_length = max([len(indexes) for indexes in indexes_batch])\n",
    "    mask2d = torch.ByteTensor(seqsToMask2D(padList))\n",
    "    return padVar, max_length, mask2d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以下函数 *pairBatch2trainData* 用于转化 sentence pair 数据：pair_batch --> input sentences 和 target sentences --> 相应的 Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pairBatch2trainData(vocabulary, pair_batch):\n",
    "    \"\"\"Returns all items for a given batch of pairs\"\"\"\n",
    "    pair_batch.sort(key=lambda x: len(x[0].split(' ')), reverse=True)  # 对pair_batch，按每个pair中第1个sentece的word数目倒序排序各pair\n",
    "    input_batch, target_batch = zip(*pair_batch)                       # 解压/拆解每个pair\n",
    "    inp, lengths = inputToTensor(input_batch, vocabulary)\n",
    "    target, max_length, mask = targetToTensor(target_batch, vocabulary)\n",
    "    return inp, lengths, target, max_length, mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> 刘尧：感悟：数据前期操作处理如向量化、padding等可在转化为Tensor之前开展，以方便操作，待所有前期操作结束后，在输入模型进行训练前，再转化为Tensor！\n",
    "\n",
    "> 刘尧：疑惑：这些前期操作，是不是在Tensor中有相应的便捷API？那么，需要先转化为Tensor？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input: tensor([[1329,   50, 3309,  787,  100],\n",
      "        [  25,   12,  758,    4,    6],\n",
      "        [ 310,  109,    4,    2,    2],\n",
      "        [  76,    4,    2,    0,    0],\n",
      "        [ 177,    2,    0,    0,    0],\n",
      "        [  53,    0,    0,    0,    0],\n",
      "        [2009,    0,    0,    0,    0],\n",
      "        [1030,    0,    0,    0,    0],\n",
      "        [  70,    0,    0,    0,    0],\n",
      "        [   2,    0,    0,    0,    0]])\n",
      "input_lengths: tensor([10,  5,  4,  3,  3])\n",
      "target: tensor([[ 116,   25, 2818, 5093, 2007],\n",
      "        [ 117,  200,    6,    2,   25],\n",
      "        [ 341,  677,    2,    0,    8],\n",
      "        [   4,   12,    0,    0,   40],\n",
      "        [   2, 4849,    0,    0,  282],\n",
      "        [   0,  227,    0,    0,    4],\n",
      "        [   0,    4,    0,    0,    4],\n",
      "        [   0,    2,    0,    0,    4],\n",
      "        [   0,    0,    0,    0,    2]])\n",
      "max_target_len: 9\n",
      "mask: tensor([[1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 0, 1],\n",
      "        [1, 1, 0, 0, 1],\n",
      "        [1, 1, 0, 0, 1],\n",
      "        [0, 1, 0, 0, 1],\n",
      "        [0, 1, 0, 0, 1],\n",
      "        [0, 1, 0, 0, 1],\n",
      "        [0, 0, 0, 0, 1]], dtype=torch.uint8)\n"
     ]
    }
   ],
   "source": [
    "# Example for validation\n",
    "inp, lengths, target, max_length, mask = pairBatch2trainData(vocabulary, [random.choice(pairs) for _ in range(5)])\n",
    "print(\"input:\", inp)\n",
    "print(\"input_lengths:\", lengths)\n",
    "print(\"target:\", target)\n",
    "print(\"max_target_len:\", max_length)\n",
    "print(\"mask:\", mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Define Models\n",
    "\n",
    "### Seq2Seq Model\n",
    "\n",
    "The goal of a seq2seq model is to take **a variable-length sequence** as an input, and return **a variable-length sequence** as an output using a fixed-sized model. By using 2 separate RNN together, we can accomplish this task. \n",
    "\n",
    "The fist RNN is an **encoder**, which encodes a variable-length input sequence to a fixed-length context vector. In theory, this context vector (the final hidden layer of the RNN) will contain semantic information about the input sequence. \n",
    "\n",
    "> **encoder**: \\[X1,X2,...,Xn\\](A variable-length sequence) --> A fixed-length context vector\n",
    "\n",
    "The second RNN is a **decoder**, which takes an input word and the context vector, and returns a guess for the next word in the sequence and a hidden state to use in the next iteration.\n",
    "\n",
    "> **decoder**: Xn(An input word) + A fixed-length context vector --> Y1(A guess for the next word) + A hidden state\n",
    "\n",
    "![001](./image/seq2seq_ts.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoder\n",
    "\n",
    "The encoder RNN iterates through the input sequence one token ***Xi*** (e.g. word) at a time, at each time step ***i*** outputting an \"output\" vector ***Yi*** and a \"hidden state\" vector ***Si***. The hidden state vector ***Si*** is then passed to the next time step ***i+1***, while the output vector ***Yi*** is recorded.\n",
    "\n",
    "> At each time step ***i***: ***Xi*** --> ***Yi*** (recorded) + ***Si*** (passed to step ***i+1***)\n",
    "\n",
    "The encoder transforms the context it saw at each point in the sequence into **a set of points in a high-dimensional space**, which the decoder will use to generate a meaningful output for the given task.\n",
    "\n",
    "The heart of encoder is a multi-layered GRU. We will use a bidirectional variant of the GRU, meaning that there are essentially 2 independent RNNs: one is fed the input sequence in normal sequential order, and the other is fed in reverse order.\n",
    "\n",
    "The outputs of each network are summed at each time step ***i***. Using a bidirectional GRU will give us the advantage of encoding both past and future context.\n",
    "\n",
    "Bidirectional RNN:\n",
    "\n",
    "![](./image/RNN-bidirectional.png)\n",
    "\n",
    "Note that:\n",
    "\n",
    "- An ***embedding layer*** is used to encode our word indices in an arbitrarily sized feature space. For our models, this layer will map each word to a feature space of size *hidden_size*.\n",
    "\n",
    "- If passing a padded batch of sequences to an RNN module, we must **pack and unpack padding** around the RNN pass using *torch.nn.utils.rnn.pack_padded_sequence* and *torch.nn.utils.rnn.pad_packed_sequence* respectively."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Computation Graph:**\n",
    "\n",
    "- Convert word indexes to embeddings\n",
    "\n",
    "- Pack padded batch of sequences for RNN module\n",
    "\n",
    "- Forward pass through GRU\n",
    "\n",
    "- Unpack padding\n",
    "\n",
    "- Sum bidirectional GRU outputs\n",
    "\n",
    "- Return output and final hidden state\n",
    "\n",
    "**Inputs:**\n",
    "\n",
    "- *input_seq*: Batch of input sentences; shape=***(max_length, batch_size)***\n",
    "\n",
    "- *input_lengths*: List of sentence lengths corresponding to each sentence in the batch; shape=***(batch_size)***\n",
    "\n",
    "- *hidden*: Hidden state; shape=***(n_layers x num_directions, batch_size, hidden_size)***\n",
    "\n",
    "**Outputs:**\n",
    "\n",
    "- *outputs*: Output features from the last hidden layer of the GRU (sum of bidirectional outputs); shape=***(max_length, batch_size, hidden_size)***\n",
    "\n",
    "- *hidden*: Updated hidden state from GRU; shape=***(n_layers x num_directions, batch_size, hidden_size)*** (same shape with *hidden* above in Inputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**几个注意点：**\n",
    "\n",
    "- embedding 位于 EncoderRNN 之外额外处理\n",
    "\n",
    "- nn.GRUCell 和 nn.GRU 有区别，前者是多个 GRUCell 组合在一起，后者才是 Encoder 中展示的那样，多个 GRU 串联在一起，可设置单向或双向，是 Encoder 的核心和主体\n",
    "\n",
    "- pack_padded_sequence 和 pad_packed_sequence 这2个函数有意思，要好好研究一下到底在干啥子\n",
    "\n",
    "- 输入中的 hidden 是所谓的 ***a0***，当n_layers=1时，Output 是 ***a1,a2,...,atx,...,aTx***，输出中的 hidden 是 ***aTx***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, embedding, n_layers=1, dropout=0):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embedding = embedding\n",
    "        \n",
    "        # Initialize GRU  注意nn.GRU与nn.GRUCell的区别！\n",
    "        # input_size和hidden_size都设置为hidden_size，因为input size is a word embedding with number of features==hidden_size\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=(0 if n_layers == 1 else dropout), bidirectional=True)\n",
    "        \n",
    "    def forward(self, input_seq, input_lengths, hidden=None):\n",
    "        embedded = self.embedding(input_seq)                                           # Convert word indexes to embeddings\n",
    "        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)      # Pack padded batch of sequences for RNN module\n",
    "        outputs, hidden = self.gru(packed, hidden)                                     # Forward pass through GRU\n",
    "        outputs, _ = torch.nn.utils.rnn.pad_packed_sequence(outputs)                   # Unpack padding\n",
    "        outputs = outputs[:, :, :self.hidden_size] + outputs[:, :, self.hidden_size:]  # Sum bidirectional GRU outpus   此时应该是对hidden进行sum吧或拼接吧？\n",
    "        return outputs, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Encoder** (来自Coursera)\n",
    "\n",
    "已知：a<0\\>,  a<Tx+1\\>, x<1\\>, x<2\\>, ..., x<tx\\>, ...x<Tx\\> 即输入sequence\n",
    "    \n",
    "流程：x<tx\\> + a<tx-1\\> --(BiLSTM/BiGRU)--> a<tx\\>, y^<tx\\>\n",
    "    \n",
    "结果：a<tx\\> = (a<tx\\>->, a<tx\\><-) 即原始单词x<tx\\>激活值，用于计算Attention和c<ty\\>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoder\n",
    "\n",
    "The decoder RNN generates the response sentence in a **token-by-token** fashion. It uses the **encoder's context vectors**, and internal hidden states to generate the next word in the sequence. It continues generating words until it outputs and EOS_TOKEN.\n",
    "\n",
    "> Token-by-token: context vectors + hidden states --> next word until EOS_TOKEN\n",
    "\n",
    "### Attention Layer\n",
    "\n",
    "A common problem with a vanilla seq2seq decoder is that if we **rely solely on the context vector** to encode the entire input sequence's meaning, it is likely that we will have information loss. This is especially the case when dealing with long input sequences, greatly limiting the capability of our decoder.\n",
    "\n",
    "Attention mechanism can combat this, which allows the decoder to **pay attention to certain parts of the input sequence**, rather than using the entire fixed context at every step.\n",
    "\n",
    "At a high level, attention is calculated using the decoder's current hidden state and the encoder's outputs (图片中是decoder's inputs ?). The output attention weights have the same shape as the input sequence, allowing us to multiply them by the encoder outputs, giving us **a weighted sum which indicates the parts of encoder output to pay attention to**.\n",
    "\n",
    "> decoder's current hidden state + decoder's inputs --> attention weights\n",
    "\n",
    "> attention weights x encoder outputs --> A weighted sum (attended encoder outputs)\n",
    "\n",
    "![](./image/attn2.png)\n",
    "\n",
    "An improvement is creating \"Global attention\". The key difference is that with \"Global attention\", we consider **all of the encoder's hidden states**, as opposed to \"Local attention\", which only considers **the encoder's hidden state from the current time step**. Another difference is that with \"Global attention\", we calculate attention weights, or energies, using the hidden state of the decoder from the current time step only. ...\n",
    "\n",
    "Overall, the \"Global attention mechanism\" can be summarized by the following figure. Note that we will implement the \"Attention Layer\" as a separate *nn.Module* called *Attn*. The output of this module is a softmax normalized weights tensor of shape *(batch_size, 1, max_length)*.\n",
    "\n",
    "![](./image/global_attn.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attn(nn.Module):\n",
    "    \"\"\"Attention Layer，Encoder所有Step的状态，对Decoder中某步ty状态的权重alpha\"\"\"\n",
    "    def __init__(self, method, hidden_size):\n",
    "        super(Attn, self).__init__()\n",
    "        if method not in ['dot', 'general', 'concat']:\n",
    "            raise ValueError(method, 'is not an appropriate attention method.')\n",
    "        self.method = method\n",
    "        self.hidden_size = hidden_size\n",
    "        if self.method == 'general':\n",
    "            self.attn_nn = nn.Linear(self.hidden_size, hidden_size)      # 小神经网络中的Dense\n",
    "        elif self.method == 'concat':\n",
    "            self.v = nn.Parameter(torch.FloatTensor(hidden_size))\n",
    "            self.attn_nn = nn.Linear(self.hidden_size * 2, hidden_size)\n",
    "    \n",
    "    def forward(self, hidden, encoder_outputs):\n",
    "        \"\"\"\n",
    "        ARG\n",
    "            hidden=decoder中的s<ty-1>，是单步ty的状态\n",
    "            encoder_outputs=encoder中的a(a<1>,a<2>,...,a<tx>,...,a<Tx>)，是所有Step的状态\n",
    "            score=score(hidden, encoder_outputs)\n",
    "        RETURN\n",
    "            alpha<ty,:>，表示Encoder所有Step的状态对单步ty的权重\n",
    "        \"\"\"\n",
    "        if self.method == 'dot':\n",
    "            score = hidden * encoder_outputs\n",
    "        elif self.method == 'general':\n",
    "            score = hidden * self.attn_nn(encoder_outputs)\n",
    "        elif self.method == 'concat':\n",
    "            cated = torch.cat((hidden.expand(encoder_outputs.size(0), -1, -1), encoder_outputs), 2)\n",
    "            score = self.v * self.attn_nn(cated).tanh()        # \n",
    "        attn_energies = torch.sum(score, dim=2).t()            # Transpose max_length and batch_size dimensions                    \n",
    "        alpha = F.softmax(attn_energies, dim=1).unsqueeze(1)   # The softmax normalized probability scores   这个softmax是小神经网络最后的Softmax\n",
    "        return alpha"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Attention** (来自coursera，只是concat模式) 与coursera中学的有一些差别！\n",
    "\n",
    "已知：s<0\\>\n",
    "\n",
    "流程：s<ty-1\\> + a<tx\\> --(Dense)--> e<ty,tx\\>  --(Softmax)--> alpha<ty,tx\\> (即Attention)\n",
    "    \n",
    "流程：alpha<ty,tx\\> + a<tx\\> --(加权之和)-->  c<ty\\>\n",
    "\n",
    "结果：c<ty\\> 即Context，用于Decoder中输入给LSTM\n",
    "\n",
    "注意：a<tx\\>两次使用：与s<ty-1\\>拼接，与alpha<ty,tx\\>加权求和\n",
    "\n",
    "注意：c<ty\\>是承上启下，承Encoder的结果，启Decoder的输入"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AttentionDecoder\n",
    "\n",
    "For the decoder, we will manually feed our batch **one time step at a time**. This means that our embedded word tensor and GRU output will both have shape ***(1, batch_size, hidden_size)***.\n",
    "\n",
    "**Computational Graph:**\n",
    "\n",
    "- Get embedding of current input word\n",
    "\n",
    "- Forward through unidirectional GRU\n",
    "\n",
    "- Calculate attention weights from the current GRU output from the step above\n",
    "\n",
    "- Multiply attention weights to encoder outputs to get new \"weighted sum\" context vector\n",
    "\n",
    "- Concatenate weighted context vector and GRU output using Luong eq. 5\n",
    "\n",
    "- Predict next word using Luong eq. 6 (without softmax)\n",
    "\n",
    "- Return output and final hidden state\n",
    "\n",
    "**Inputs:**\n",
    "\n",
    "- *input_step*: **one time step (one word)** of input sequence batch; shape=***(1, batch_size)***\n",
    "\n",
    "- *last_hidden*: final hidden layer of GRU; shape=*(n_layers x num_directions, batch_size, hidden_size)*\n",
    "\n",
    "- *encoder_outputs*: encoder model's output; shape=*(max_length, batch_size, hidden_size)*\n",
    "\n",
    "**Outputs:**\n",
    "\n",
    "- *output*: softmax normalized tensor giving probabilities of each word being the correct next word in the decoded sequence; shape=*(batch_size, vocabulary.num_words)*\n",
    "\n",
    "- *hidden*: final hidden state of GRU; shape=*(n_layers x num_directions, batch_size, hidden_size)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LuongAttnDecoderRNN(nn.Module):\n",
    "    def __init__(self, attn_method, embedding, hidden_size, output_size, n_layers=1, dropout=0.1):\n",
    "        super(LuongAttnDecoderRNN, self).__init__()\n",
    "        \n",
    "        # Keep for reference\n",
    "        self.attn_method = attn_method\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        # Define layers\n",
    "        self.embedding = embedding\n",
    "        self.embedding_dropout = nn.Dropout(dropout)          # 干啥子用的？\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size, n_layers, dropout=(0 if n_layers == 1 else dropout))  # Decoder中的一个RNN Cell???\n",
    "        self.attn = Attn(attn_method, hidden_size)            # Attention Layer 使用在此！\n",
    "        self.linear1 = nn.Linear(hidden_size * 2, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "    # Note: we run this one step (word) at a time\n",
    "    def forward(self, input_step, last_hidden, encoder_outputs):\n",
    "        # Get embedding of current input word\n",
    "        embedded = self.embedding(input_step)\n",
    "        embedded = self.embedding_dropout(embedded)\n",
    "        \n",
    "        rnn_output, hidden = self.gru(embedded, last_hidden)         # Forward through GRU\n",
    "        attn_weights = self.attn(rnn_output, encoder_outputs)        # Calculate attention weights from the current GRU output\n",
    "        context = attn_weights.bmm(encoder_outputs.transpose(0, 1))  # Multiply attention weights to encoder outputs\n",
    "        \n",
    "        # Concatenate weighted context vector and GRU output using Luong eq. 5\n",
    "        concat_input = torch.cat((rnn_output.squeeze(0), context.squeeze(1)), 1)  # 为什么之后没有再输入到下一个GRU中？？？\n",
    "        concat_output = torch.tanh(self.linear1(concat_input))\n",
    "        \n",
    "        # Predict next word using Luong eq. 6\n",
    "        output = self.linear2(concat_output)\n",
    "        output = F.softmax(output, dim=1)\n",
    "        return output, hidden      # hidden随后在下一Step时作为last_hidden继续输入至LuongAttnDecoderRNN中"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Define Training Procedure\n",
    "\n",
    "### Masked loss\n",
    "\n",
    "Since we are dealing with batches of padded sequences, we can't simply consider all elements of the tensor when calculating loss. We define *maskNLLLoss* to calculate loss based on decoder's output tensor, the target tensor, and a binary mask tensor **describing the padding of the target tensor**. This function calculates the average negative log likelihood of **the elements that corresponding to a 1 in the mask tensor**.\n",
    "\n",
    "> 刘尧：嘛意思？为什么要有mask！？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maskNLLLoss(inp, target, mask):\n",
    "    crossEntropy = - torch.log(torch.gather(inp, 1, target.view(-1, 1)).squeeze(1))\n",
    "    loss = crossEntropy.masked_select(mask).mean()\n",
    "    loss = loss.to(device)\n",
    "    return loss, mask.sum().item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Single training iteration\n",
    "\n",
    "Single training iteration means **a single batch** of inputs.\n",
    "\n",
    "We will use 2 clever tricks to aid in convergence:\n",
    "\n",
    "- **Teacher forcing**: At some probability, set by *teacher_forcing_ratio*, we use the **current target word** as the decoder's next input rather than using the **decoder's current guess**. This technique aids in more efficient training. However, it can lead to model **instability during inference**, as the decoder may not have a sufficient chance to truly craft its own output sequences during training.\n",
    "\n",
    "- **Gradient clipping**: This is a commonly used technique for countering the \"exploding gradient\" problem. In essence, by clipping or thresholding gradients to a maximum value, we prevent the gradients from **growing exponentially and either overflow (NaN), or overshoot steep cliffs in the cost function**.\n",
    "\n",
    "**Sequence of Operations:**\n",
    "\n",
    "1. Forward pass entire input batch through encoder.\n",
    "\n",
    "2. Initialize decoder inputs as SOS_TOKEN, and hidden state as the encoder's final hidden state.\n",
    "\n",
    "3. Forward input batch sequence through decoder one time step at a time.\n",
    "\n",
    "4. If teacher forcing: set the current target as next decoder input; else: set current decoder output as next decoder input.\n",
    "\n",
    "5. Calculate and accumulate loss.\n",
    "\n",
    "6. Perform backpropagation.\n",
    "\n",
    "7. Clip gradients.\n",
    "\n",
    "8. Update encoder and decoder model parameters.\n",
    "\n",
    "> 刘尧：虽然实际上，对于RNN Modules (RNN, LSTM, GRU)，我们 **one time-step at a time** 地迭代着处理，每次都计算当前的 hidden state，然后开始下一次迭代，但我们仍可像对待 non-recurrent layers 那样，**一次性输入 entire input sequence (or batch of sequences)**。当然，我们也可手动进行每次的迭代，就像在 decoder 中做的那样。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(inp, lengths, target, mask, max_target_len, encoder, decoder, embedding, \n",
    "          encoder_optimizer, decoder_optimizer, batch_size, clip, max_length=MAX_LENGTH):\n",
    "    # Zero gradients\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "    \n",
    "    # Set device options\n",
    "    inp = inp.to(device)\n",
    "    lengths = lengths.to(device)\n",
    "    target = target.to(device)\n",
    "    mask = mask.to(device)\n",
    "    \n",
    "    # Initialize variables\n",
    "    loss = 0\n",
    "    print_losses = []\n",
    "    n_totals = 0\n",
    "    \n",
    "    # 1. Forward pass through encoder\n",
    "    encoder_outputs, encoder_hidden = encoder(inp, lengths)\n",
    "    \n",
    "    # 2.1 Initialize decoder input (start with SOS_TOKEN for each sentence)\n",
    "    decoder_input = torch.LongTensor([[SOS_TOKEN for _ in range(batch_size)]])  # batch中每个sequence都是SOS_TOKEN\n",
    "    decoder_input = decoder_input.to(device)\n",
    "    \n",
    "    # 2.2 Initialize decoder hidden state to the encoder's final hidden state\n",
    "    decoder_hidden = encoder_hidden[: decoder.n_layers]  # n_layers ???\n",
    "    \n",
    "    # 4.1 Determine if we are using teacher forcing this iteration\n",
    "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "    \n",
    "    # 3. Forward batch of sequences through decoder one time step at a time\n",
    "    if use_teacher_forcing:\n",
    "        for t in range(max_target_len):  # max_target_len表示共有多少个time step\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            \n",
    "            # 4.2 Teacher forcing: next input is current target\n",
    "            decoder_input = target[t].view(1, -1)\n",
    "            \n",
    "            # 5. Calculate and accumulate loss\n",
    "            mask_loss, nTotal = maskNLLLoss(decoder_output, target[t], mask[t])\n",
    "            loss += mask_loss\n",
    "            print_losses.append(mask_loss.item() * nTotal)\n",
    "            n_totals += nTotal\n",
    "    else:\n",
    "        for t in range(max_target_len):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            \n",
    "            # 4.2 No teacher forcing: next input is decoder's own current output, that is, its guess ?\n",
    "            _, topi = decoder_output.topk(1)\n",
    "            decoder_input = torch.LongTensor([[topi[i][0] for i in range(batch_size)]])\n",
    "            decoder_input = decoder_input.to(device)  # 为什么上面没有？\n",
    "            \n",
    "            # 5. Calculate and accumulate loss\n",
    "            mask_loss, nTotal = maskNLLLoss(decoder_output, target[t], mask[t])\n",
    "            loss += mask_loss\n",
    "            print_losses.append(mask_loss.item() * nTotal)\n",
    "            n_totals += nTotal\n",
    "            \n",
    "    # 6. Perform backpropagation\n",
    "    loss.backward()\n",
    "    \n",
    "    # 7. Clip gradients: gradients are modified in place\n",
    "    _ = nn.utils.clip_grad_norm_(encoder.parameters(), clip)\n",
    "    _ = nn.utils.clip_grad_norm_(decoder.parameters(), clip)\n",
    "    \n",
    "    # 8. Adjust model weights\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "    \n",
    "    return sum(print_losses) / n_totals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training iteration\n",
    "\n",
    "It's finally time to tie the full training procedure together with the data. \n",
    "\n",
    "One thing to note is that when we save model, we save a tarball containing **the encoder and decoder state_dicts(parameters), the optimizer's state_dicts, the loss, the iteration**, etc. \n",
    "\n",
    "Saving model in this way will give us the ultimate flexibility with the checkpoint. After loading a checkpoint, we will be able to use the model parameters to **run inference, or continue training right where we left off**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainIters(model_name, vocabulary, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer, embedding, encoder_n_layers, \n",
    "               decoder_n_layers, save_dir, n_iterations, batch_size, print_every, save_every, clip, corpus, checkpoint_file):\n",
    "    \n",
    "    # Load batches for each iteration\n",
    "    training_batches = [pairBatch2trainData(vocabulary, [random.choice(pairs) for _ in range(batch_size)]) for _ in range(n_iterations)]\n",
    "    \n",
    "    # Initializations\n",
    "    print('Initializing ...')\n",
    "    next_iteration = 0\n",
    "    print_loss = 0\n",
    "    if checkpoint_file:\n",
    "        # 难道checkpoint只在此处使用？只用到iteration这一个信息？？？不需要获得model和optimizer的state_dict然后基于此继续训练吗？？\n",
    "        checkpoint = torch.load(checkpoint_file)\n",
    "        next_iteration = checkpoint['iteration'] + 1\n",
    "        # 此处不需要从checkpoint中获取encoder,decoder,encoder_optimizer,decoder_optimizer,embedding等吗！？！比如encoder.load_state_dict(checkpoint['encoder'])\n",
    "        \n",
    "    # Training loop\n",
    "    print('Training ...')\n",
    "    for i in range(next_iteration, n_iterations):\n",
    "        training_batch = training_batches[i]\n",
    "        inp, lengths, target, max_target_len, mask = training_batch  # Extract fields from batch\n",
    "        \n",
    "        # Run a training interation with batch\n",
    "        loss = train(inp, lengths, target, mask, max_target_len, encoder, decoder, embedding, encoder_optimizer, decoder_optimizer, batch_size, clip)\n",
    "        print_loss += loss\n",
    "        \n",
    "        # Print progress\n",
    "        if (i + 1) % print_every == 0:\n",
    "            print_loss_avg = print_loss / print_every\n",
    "            print(f'Iteration: {i + 1}; Percent complete: {(i + 1) / n_iterations: .1f}%; Average loss: {print_loss_avg: .4f}')\n",
    "            print_loss = 0\n",
    "        \n",
    "        # Save checkpoint\n",
    "        if (i + 1) % save_every == 0:\n",
    "            directory = os.path.join(save_dir, model_name, corpus_name, f'{encoder_n_layers}-{decoder_n_layers}_{hidden_size}')\n",
    "            if not os.path.exists(directory):\n",
    "                os.makedirs(directory)\n",
    "            torch.save({\n",
    "                'iteration': i,\n",
    "                'encoder': encoder.state_dict(),\n",
    "                'decoder': decoder.state_dict(),\n",
    "                'encoder_optimizer': encoder_optimizer.state_dict(),\n",
    "                'decoder_optimizer': decoder_optimizer.state_dict(),\n",
    "                'loss': loss,\n",
    "                'voc_dict': vocabulary.__dict__,\n",
    "                'embedding': embedding.state_dict()\n",
    "            }, os.path.join(directory, f'{i}_checkpoint.tar'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Define Inference\n",
    "\n",
    "We should define how we want the model to decode the encoded input.\n",
    "\n",
    "### Greedy decoding\n",
    "\n",
    "Greedy decoding is the decoding method that we use during training when we are **NOT** using teacher forcing. In other words, for each time step, we simply choose **the word from decoder_output with the highes softmax value**. This decoding method is optimal on a single time-step level.\n",
    "\n",
    "To facilite the greedy decoding operation, we define a *GreedySearchDecoder* class. When run, an object of this class takes an input sequence (*input_seq*) of shape *(input_seq length, 1)*, a scalar input length *(input_length)* tensor, and a *max_length* to bound the response sentence length.\n",
    "\n",
    "The input sentence is inferenced using the following computational graph:\n",
    "\n",
    "**Computational Graph:**\n",
    "\n",
    "1. Forward input through encoder model\n",
    "\n",
    "2. Prepare encoder's final hidden layer to be first hidden input to the decoder\n",
    "\n",
    "3. Initialize decoder's first input as SOS_TOKEN\n",
    "\n",
    "4. Initialize tensors to append decoded words to\n",
    "\n",
    "5. Iteratively decode one word token at a time:\n",
    "\n",
    "  5.1 Forward pass through decoder\n",
    "  \n",
    "  5.2 Obtain most likely word token and its softmax score\n",
    "  \n",
    "  5.3 Record token and score\n",
    "  \n",
    "  5.4 Prepare current token to be next decoder input\n",
    "  \n",
    "6. Return collections of word tokens and scores\n",
    "\n",
    "> 刘尧：疑问：这又是要干啥子玩意儿？？这么多Decoder啊？另外，decoder到底是指整个Decoder模块，还是Decoder模块中每个time step对应的GRU/LSTM Cell？？？\n",
    "\n",
    "> 刘尧：只是用于应用环节！即：计算decoder的所有输出以及对应的scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GreedySearchDecoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, encoder, decoder):\n",
    "        super(GreedySearchDecoder, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        \n",
    "    def forward(self, input_seq, input_length, max_length):\n",
    "        # 1. Forward input through encoder model\n",
    "        encoder_outputs, encoder_hidden = self.encoder(input_seq, input_length)\n",
    "        \n",
    "        # 2. Prepare encoder's final hidden layer to be first hidden input to the decoder\n",
    "        decoder_hidden = encoder_hidden[: decoder.n_layers]\n",
    "        \n",
    "        # 3. Initialize decoder input with SOS_TOKEN\n",
    "        decoder_input = torch.ones(1, 1, device=device, dtype=torch.long) * SOS_TOKEN\n",
    "        \n",
    "        # 4. Initialize tensors to append decoded words to\n",
    "        all_tokens = torch.zeros([0], device=device, dtype=torch.long)\n",
    "        all_scores = torch.zeros([0], device=device)\n",
    "        \n",
    "        # 5. Iteratively decode one word token at a time\n",
    "        for _ in range(max_length):\n",
    "            # 5.1 Forward pass through decoder\n",
    "            decoder_output, decoder_hidden = self.decoder(decoder_input, decoder_hidden, encoder_outputs)  # 疑问：此处的decoder应该是LSTM/GRU Cell吧？\n",
    "            \n",
    "            # 5.2 Obtain most likely word token and its softmax value\n",
    "            decoder_scores, decoder_input = torch.max(decoder_output, dim=1)\n",
    "            \n",
    "            # 5.3 Record token and score\n",
    "            all_tokens = torch.cat((all_tokens, decoder_input), dim=0)\n",
    "            all_scores = torch.cat((all_scores, decoder_scores), dim=0)\n",
    "            \n",
    "            # 5.4 Prepare current token to be next decoder input\n",
    "            decoder_input = torch.unsqueeze(decoder_input, 0)\n",
    "        \n",
    "        # 6. Return collections of word tokens and scores\n",
    "        return all_tokens, all_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference my text\n",
    "\n",
    "Having decoding method defined, we can write function ***inference*** for inferencing a string input sentence. *inference* manages the *low-level process* of handling the input sentence.\n",
    "\n",
    "We first format the sentence as an input batch of word indexes with batch_size=1 by converting the words of the sentence to their corresponding indexes, and transposing the dimensions to prepare the tensor for our models. We also create a *lengths* tensor which contains the length of our input sentence. In this case, *length* is scalar because we are only evaluating one sentence at a time (batch_size==1).\n",
    "\n",
    "Next, we obtain the decoded response sentence tensor using our *GreedySearchDecoder* object (*searcher*).\n",
    "\n",
    "Finally, we convert the response's indexes to words and return the list of decoded words.\n",
    "\n",
    "> **inference**: words of one sentence --> their corresponding indexes --transpose-->  tensor for models  --GreedySearchDecoder--> decoded response indexes --> list of decoded words\n",
    "\n",
    "***inferenceInput*** acts as the user interface for our chatbot. When called, an input sentence is **normalized in the same way as training data**, and is ultimately fed to the **inference** function to obtain a decoded output sentence. We loop this process, so we can keep chatting until we enter either 'q' or 'quit'.\n",
    "\n",
    "Finally, if a sentence contains a OOV word, we handle this gracefully by printing an error message and prompting the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inference(encoder, decoder, searcher, vocabulary, sentence, max_length=MAX_LENGTH):\n",
    "    # words --> indexes\n",
    "    indexes_batch = [sentenceToIndexes(vocabulary, sentence)]\n",
    "    lengths = torch.tensor([len(indexes) for indexes in indexes_batch])\n",
    "    lengths = lengths.to(device)\n",
    "    \n",
    "    # Transpose dimensions of batch for models\n",
    "    input_batch = torch.LongTensor(indexes_batch).transpose(0, 1)\n",
    "    input_batch = input_batch.to(device)\n",
    "    \n",
    "    # Decode sentence with searcher\n",
    "    tokens, _ = searcher(input_batch, lengths, max_length)\n",
    "    \n",
    "    # indexes --> words\n",
    "    decoded_words = [vocabulary.index2word[token.item()] for token in tokens]\n",
    "    \n",
    "    return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inferenceInput(encoder, decoder, searcher, vocabulary):\n",
    "    input_sentence = ''\n",
    "    while(1):\n",
    "        try:\n",
    "            # Get input sentence\n",
    "            input_sentence = input('>')\n",
    "            if input_sentence in ('q', 'quit'):\n",
    "                break\n",
    "                \n",
    "            # Normalize sentence\n",
    "            input_sentence = normalizeString(input_sentence)\n",
    "            \n",
    "            # Inference sentence\n",
    "            output_words = inference(encoder, decoder, searcher, vocabulary, input_sentence)\n",
    "            \n",
    "            # Format and print response sentence\n",
    "            output_words[:] = [x for x in output_words if not (x in ('EOS', 'PAD'))]\n",
    "            print('Bot:', ' '.join(output_words))\n",
    "            \n",
    "        except KeyError:\n",
    "            print('Error: Encountered unkonw word.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Run Model\n",
    "\n",
    "Regardless of whether we want to train or test the chatbot model, we must **initialize the individual encoder and decoder models**. \n",
    "\n",
    "In the following block, we set our desired configurations, choose to start from scratch or set a checkpoint to load from, and build and initialize the models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configure models\n",
    "model_name = 'cb_model'\n",
    "attn_method = 'dot'\n",
    "#attn_method = 'general'\n",
    "#attn_method = 'concat'\n",
    "hidden_size = 500\n",
    "encoder_n_layers = 2\n",
    "decoder_n_layers = 2\n",
    "dropout = 0.1\n",
    "batch_size = 64\n",
    "\n",
    "# Set checkpoint to load from; set to None if starting from scratch\n",
    "checkpoint_file = None\n",
    "checkpoint_iter = 4000\n",
    "#checkpoint_file = os.path.join(save_dir, model_name, corpus_name, f'{encoder_n_layers}-{decoder_n_layers}_{hidden_size}', f'{checkpoint_iter}_checkpoint.tar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model if a checkpoint_file is provided\n",
    "if checkpoint_file:\n",
    "    checkpoint = torch.load(checkpoint_file)\n",
    "    #checkpoint = torch.load(checkpoint_file, map_location=torch.device('cpu'))  # If loading a model trained on GPU to CPU\n",
    "    encoder_sd = checkpoint['encoder']\n",
    "    decoder_sd = checkpoint['decoder']\n",
    "    encoder_optimizer_sd = checkpoint['encoder_optimizer']\n",
    "    decoder_optimizer_sd = checkpoint['decoder_optimizer']\n",
    "    embedding_sd = checkpoint['embedding']\n",
    "    vocabulary.__dict__ = checkpoint['voc_dict']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Embedding, Encoder, Decoder三者分别独立定义，使用时再结合在一起。训练和应用时一起进行，又可独立拿出来。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building encoder and decoder ...\n",
      "Models built and ready to go!\n"
     ]
    }
   ],
   "source": [
    "print('Building encoder and decoder ...')\n",
    "\n",
    "# Initialize word embeddings\n",
    "embedding = nn.Embedding(vocabulary.num_words, hidden_size)\n",
    "if checkpoint_file:\n",
    "    embedding.load_state_dict(embedding_sd)\n",
    "    \n",
    "# Initialize encoder & decoder models\n",
    "encoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout)\n",
    "decoder = LuongAttnDecoderRNN(attn_method, embedding, hidden_size, vocabulary.num_words, decoder_n_layers, dropout)\n",
    "if checkpoint_file:\n",
    "    encoder.load_state_dict(encoder_sd)\n",
    "    decoder.load_state_dict(decoder_sd)\n",
    "    \n",
    "encoder = encoder.to(device)\n",
    "decoder = decoder.to(device)\n",
    "print('Models built and ready to go!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LuongAttnDecoderRNN(\n",
       "  (embedding): Embedding(7826, 500)\n",
       "  (embedding_dropout): Dropout(p=0.1)\n",
       "  (gru): GRU(500, 500, num_layers=2, dropout=0.1)\n",
       "  (attn): Attn()\n",
       "  (concat): Linear(in_features=1000, out_features=500, bias=True)\n",
       "  (out): Linear(in_features=500, out_features=7826, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Configure training/optimization\n",
    "clip = 50.0\n",
    "teacher_forcing_ratio = 1.0\n",
    "learning_rate = 0.0001\n",
    "decoder_learning_ratio = 5.0\n",
    "n_iteration = 4000\n",
    "print_every = 1\n",
    "save_every = 500\n",
    "\n",
    "# Ensure dropout layers are in train mode\n",
    "encoder.train()\n",
    "decoder.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building optimizers ...\n"
     ]
    }
   ],
   "source": [
    "# Initialize optimizers\n",
    "print('Building optimizers ...')\n",
    "encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)\n",
    "decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate * decoder_learning_ratio)\n",
    "if checkpoint_file:\n",
    "    encoder_optimizer.load_state_dict(encoder_optimizer_sd)\n",
    "    decoder_optimizer.load_state_dict(decoder_optimizer_sd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Training!\n",
      "Initializing ...\n",
      "Training ...\n",
      "Iteration: 1; Percent complete:  0.0%; Average loss:  8.9690\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-fe0c7fbb0377>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Starting Training!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m trainIters(model_name, vocabulary, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer, embedding, encoder_n_layers, decoder_n_layers, \n\u001b[0;32m----> 4\u001b[0;31m            save_dir, n_iteration, batch_size, print_every, save_every, clip, corpus_name, checkpoint_file)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-32-9a547e880c72>\u001b[0m in \u001b[0;36mtrainIters\u001b[0;34m(model_name, vocabulary, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer, embedding, encoder_n_layers, decoder_n_layers, save_dir, n_iterations, batch_size, print_every, save_every, clip, corpus, checkpoint_file)\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;31m# Run a training interation with batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_target_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoder_optimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder_optimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0mprint_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-31-dd8a1d873b55>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(inp, lengths, target, mask, max_target_len, encoder, decoder, embedding, encoder_optimizer, decoder_optimizer, batch_size, clip, max_length)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;31m# 8. Adjust model weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m     \u001b[0mencoder_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m     \u001b[0mdecoder_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     99\u001b[0m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m                 \u001b[0mbias_correction1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Run training iterations\n",
    "print(\"Starting Training!\")\n",
    "trainIters(model_name, vocabulary, pairs, encoder, decoder, encoder_optimizer, decoder_optimizer, embedding, encoder_n_layers, decoder_n_layers, \n",
    "           save_dir, n_iteration, batch_size, print_every, save_every, clip, corpus_name, checkpoint_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run Inference\n",
    "\n",
    "To chat with your model, run the following block."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set dropout layers to eval mode\n",
    "encoder.eval()\n",
    "decoder.eval()\n",
    "# Initialize search module\n",
    "searcher = GreedySearchDecoder(encoder, decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Begin chatting\n",
    "inferenceInput(encoder, decoder, searcher, vocabulary)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
